package ca.uwindsor.cs.deepweb.estimation.experiment.service;

import java.io.IOException;
import java.io.StringWriter;
import java.util.ArrayList;
import java.util.Set;

import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.search.Hits;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.Sort;

import au.com.bytecode.opencsv.CSVWriter;
import ca.uwindsor.cs.deepweb.estimation.DataCollector;
import ca.uwindsor.cs.deepweb.estimation.DataCollectorWithLimitationRandom;
import ca.uwindsor.cs.deepweb.estimation.DataCollectorWithLimitationTop;
import ca.uwindsor.cs.deepweb.estimation.FilesContentIndexer;
import ca.uwindsor.cs.deepweb.estimation.experiment.QueryRunner;
import ca.uwindsor.cs.deepweb.estimation.method.border.MatrixLimited;
import ca.uwindsor.cs.deepweb.utility.StandardDeviation;

public class QueryServiceBordersTwoSetsCorrected extends QueryRunner {

	// protected int stepsize;

	protected ArrayList<String[]> arraylist_record;

	protected String sizeindex;
	protected String docuniquetermsindex;

	protected boolean isRepeatedDocumentIndex;

	protected String uniqueIDFieldName;

	protected final String[] terms_; //

	private int unique;
	private long accumulcativetotal;

	private float precentagecovered;
	private float OR;
	private int duplicatedfromlastquery;
	private int estimatedsize;
	private String[] result;

	private boolean isOutputORmethodResult;
	private boolean isOutputTermWeightAndNumofDoc;
	private boolean isOutputTermWeightDistribution;
	private boolean isOutputDocumentCaptureFrequencyDistribution;
	private boolean isOutputDocumentSizeDistribution;
	private boolean isOutputDocumentFrequency;
	private boolean isOutputDocumentSize;
	private boolean isOutputDocumentUniqueAndTotalTerms;
	private boolean isOutputDocumentWeight;
	private boolean isLimitedReturnTop;

	private int sortmethod;

	private boolean isReverseSorting;

	private float exponentofOR;
	
	protected DataCollector datacollectorB;
	
	private double meanweight;
	
	/**
	 * @param indexdir
	 * @param corpussize
	 * @param termslistA 
	 * @param termslistB 
	 * @param stepsize
	 * @param sizeindexdir
	 * @param isRepeated
	 * @throws IOException 
	 */
	public QueryServiceBordersTwoSetsCorrected(String indexdir, int corpussize,
			String[] termslistA, String[] termslistB, int stepsize) throws IOException {
		super(indexdir, corpussize, termslistA, null, "0", null);
		// TODO Auto-generated constructor stub
		this.stepsize = stepsize;
		arraylist_record = new ArrayList<String[]>();
		terms_ = termslistB;
		sizeindex = null;
		isRepeatedDocumentIndex = false;
		if (isRepeatedDocumentIndex) {
			uniqueIDFieldName = FilesContentIndexer.FIELD_INTERNAL_ID;
		} else {
			uniqueIDFieldName = FilesContentIndexer.FIELD_ID;
		}
		isOutputTermWeightAndNumofDoc = false;
		isOutputORmethodResult = true;
		isOutputDocumentCaptureFrequencyDistribution = true;
		isOutputDocumentSizeDistribution = true;
		isOutputDocumentFrequency = false;
		isOutputDocumentSize = false;
		isOutputDocumentUniqueAndTotalTerms = false;
		isOutputDocumentWeight = false;
		isLimitedReturnTop = false;
		sortmethod = 0;
		isReverseSorting = false;
		exponentofOR = -2.2f / 2;
	}

	protected void initialize() {
		queryparser = new QueryParser(FilesContentIndexer.FIELD_CONTENT,
				new StandardAnalyzer());
	}

	public String goQuery() throws Exception {
		datacollector = isLimitedReturnTop ? new DataCollectorWithLimitationTop(
				uniqueIDFieldName, stepsize)
				: new DataCollectorWithLimitationRandom(uniqueIDFieldName,
						stepsize);
		datacollector.setAnalyzeDocumentFrequency(false);
		
		datacollectorB = isLimitedReturnTop ? new DataCollectorWithLimitationTop(
				uniqueIDFieldName, stepsize)
				: new DataCollectorWithLimitationRandom(uniqueIDFieldName,
						stepsize);
		datacollectorB.setAnalyzeDocumentFrequency(false);
		
		MatrixLimited mwordsAB = new MatrixLimited(uniqueIDFieldName, termslist);

		IndexSearcher searcher = new IndexSearcher(indexDir);
		Hits hitsA = null;
		Query queryA = null;
		Hits hitsB = null;
		Query queryB = null;
		int listlength = termslist.length;
		int i;
		Set<String> newidsA;
		Set<String> newidsB;
		boolean isCalculateDF = (isOutputDocumentFrequency
				|| isOutputDocumentCaptureFrequencyDistribution
				|| isOutputTermWeightAndNumofDoc
				|| isOutputTermWeightDistribution || isOutputDocumentWeight) ? true
				: false;
		for (i = 0; i < listlength; i++) {
			queryA = queryparser.parse(termslist[i].trim());
			queryB = queryparser.parse(terms_[i].trim());
			switch (sortmethod) {
			case DataCollector.SORT_COMPUTED_RELEVANCE:
				hitsA = searcher.search(queryA, new Sort());
				break;
			case DataCollector.SORT_DOCUMENT_ID:
				hitsA = searcher.search(queryA, new Sort(
						FilesContentIndexer.FIELD_ID, isReverseSorting));
				break;
			case DataCollector.SORT_DOCUMENT_SIZE:
				hitsA = searcher.search(queryA, new Sort(
						FilesContentIndexer.FIELD_SIZE, isReverseSorting));
				break;
			case DataCollector.SORT_DOCUMENT_TITLE:
				hitsA = searcher.search(queryA, new Sort(
						FilesContentIndexer.FIELD_TITLE, isReverseSorting));
				break;
			default:
				hitsA = searcher.search(queryA);
				hitsB = searcher.search(queryB);
				break;
			}
			newidsA = datacollector.add(hitsA);
			newidsB = datacollectorB.add(hitsB);
			if (isOutputORmethodResult) {
				generateORmethodResult(termslist[i]);
			}
			if (isCalculateDF) {
				mwordsAB.addQuery(i, newidsA, newidsB);
			}
		}
		searcher.close();

		Set<String> DAids=datacollector.getUniqueIDs();
		Set<String> DBids=datacollectorB.getUniqueIDs();
		
		if (isCalculateDF) {
			mwordsAB.calculateInverseDocumentWeight();
		}
		String[] record = new String[2];

		/* create xml file */
		StringBuffer resultxml = new StringBuffer();

		// Java CSV to store OR method result
		StringWriter stringwriter = new StringWriter();
		CSVWriter csvwriter = new CSVWriter(stringwriter);
		String[] head;

		// Output Term weight and its frequency
		if (isOutputTermWeightAndNumofDoc || isOutputTermWeightDistribution) {
			mwordsAB.calculateQueryWeight(DAids,DBids);
			int num = 0;
			float w;
			////
			float[] qw;

			record = new String[3];
			if (isOutputTermWeightAndNumofDoc) {
				head = new String[] { "Term", "Weight", "DocumentFrequency" };
				stringwriter = new StringWriter();
				csvwriter = new CSVWriter(stringwriter);
				csvwriter.writeNext(head);
			}
			
			////
			qw = mwordsAB.getQueryWeight();
			double[] wt = new double[qw.length];
			for (i=0;i<qw.length;i++){
				wt[i]=qw[i];
			}
			StandardDeviation.sdKnuth(wt);
			meanweight=StandardDeviation.avg;
			
			for (i = 0; i < termslist.length; i++) {
				w = qw[i];
				if (isOutputTermWeightAndNumofDoc) {
					record[0] = termslist[i];
					record[1] = Float.toString(w);
					num = mwordsAB.getDocumentFrequencybyTerm(i);
					record[2] = Integer.toString(num);
					csvwriter.writeNext(record);
				}
			}
			csvwriter.flush();
			resultxml.append(stringwriter.toString());
			csvwriter.close();
			stringwriter.close();
		}

		/* finalize */

		return resultxml.toString();
	}

	protected void generateORmethodResult(String word) {
		unique = datacollector.getUnique();
		accumulcativetotal = datacollector.getTotalItems();
		duplicatedfromlastquery = datacollector
				.getDuplicatedItemsCompareToLastQuery();

		if (unique == 0) {
			OR = 0;
		} else {
			OR = accumulcativetotal / (float) unique;
		}
		precentagecovered = (float) unique / corpus_size;

		estimatedsize = (int) Double.doubleToLongBits(Double
				.longBitsToDouble(unique)
				/ (1 - Math.pow(OR, exponentofOR)));

		result = new String[9];

		result[0] = word;
		result[1] = Integer.toString(unique);
		result[2] = Long.toString(accumulcativetotal);
		result[3] = Float.toString(OR);
		result[4] = Float.toString(precentagecovered);
		if (OR == 0.0) {
			result[5] = "-";
		} else {
			result[5] = Integer.toString(estimatedsize);
		}
		result[6] = Integer.toString(datacollector.getReturnedItems());
		result[7] = Integer.toString(duplicatedfromlastquery);
		result[8] = Integer.toString(datacollector.getMarkedItems());

		arraylist_record.add(result);
	}

	/**
	 * @return The data collector
	 */
	public Set<String> getUniqueIDSet() {
		return datacollector.getUniqueIDs();
	}

	/**
	 * @param indexdir
	 *            the document and its number of unique terms index directory
	 */
	public void setOuputDocumentUniqueAndTotalTerms(String indexdir) {
		if (indexdir == null) {
			this.isOutputDocumentUniqueAndTotalTerms = false;
		} else {
			this.isOutputDocumentUniqueAndTotalTerms = true;
			docuniquetermsindex = indexdir;
		}
	}

	/**
	 * @return the isOutputDocumentUniqueAndTotalTerms
	 */
	public boolean isOuputDocumentUniqueAndTotalTerms() {
		return isOutputDocumentUniqueAndTotalTerms;
	}

	/**
	 * @param isOutputDocumentCaptureFrequencyDistribution
	 *            the isOutputDocumentCaptureFrequencyDistribution to set
	 */
	public void setOutputDocumentCaptureFrequencyDistribution(
			boolean isOutputDocumentCaptureFrequencyDistribution) {
		this.isOutputDocumentCaptureFrequencyDistribution = isOutputDocumentCaptureFrequencyDistribution;
	}

	/**
	 * @return the isOutputDocumentCaptureFrequencyDistribution
	 */
	public boolean isOutputDocumentFrequencyDistribution() {
		return isOutputDocumentCaptureFrequencyDistribution;
	}

	/**
	 * @param isOutputDocumentSizeDistribution
	 *            the isOutputDocumentSizeDistribution to set
	 */
	public void setOutputDocumentSizeDistribution(
			boolean isOutputDocumentSizeDistribution) {
		this.isOutputDocumentSizeDistribution = isOutputDocumentSizeDistribution;
	}

	/**
	 * @return the isOutputDocumentSizeDistribution
	 */
	public boolean isOutputDocumentSizeDistribution() {
		return isOutputDocumentSizeDistribution;
	}

	/**
	 * @param isOutputDocumentSize
	 *            the isOutputDocumentSize to set
	 */
	public void setOutputDocumentSize(boolean isOutputDocumentSize) {
		this.isOutputDocumentSize = isOutputDocumentSize;
	}

	/**
	 * @return the isOutputDocumentSize
	 */
	public boolean isOutputDocumentSize() {
		return isOutputDocumentSize;
	}

	/**
	 * @param isOutputDocumentFrequency
	 *            the isOutputDocumentFrequency to set
	 */
	public void setOutputDocumentFrequency(boolean isOutputDocumentFrequency) {
		this.isOutputDocumentFrequency = isOutputDocumentFrequency;
	}

	/**
	 * @return the isOutputDocumentFrequency
	 */
	public boolean isOutputDocumentFrequency() {
		return isOutputDocumentFrequency;
	}

	/**
	 * @param isOutputORmethodResult
	 *            the isOutputORmethodResult to set
	 */
	public void setOutputORmethodResult(boolean isOutputORmethodResult) {
		this.isOutputORmethodResult = isOutputORmethodResult;
	}

	/**
	 * @return the isOutputORmethodResult
	 */
	public boolean isOutputORmethodResult() {
		return isOutputORmethodResult;
	}

	/**
	 * @param isOutputTermWeightAndNumofDoc
	 *            the isOutputTermWeightAndNumofDoc to set
	 */
	public void setOutputTermWeightandNumberofDocument(
			boolean isOutputTermWeightAndNumofDoc) {
		this.isOutputTermWeightAndNumofDoc = isOutputTermWeightAndNumofDoc;
	}

	/**
	 * @return the isOutputTermWeightAndNumofDoc
	 */
	protected boolean isOutputTermWeightandDF() {
		return isOutputTermWeightAndNumofDoc;
	}

	/**
	 * @param isOutputTermWeightDistribution
	 *            the isOutputTermWeightDistribution to set
	 */
	public void setOutputTermWeightDistribution(
			boolean isOutputTermWeightDistribution) {
		this.isOutputTermWeightDistribution = isOutputTermWeightDistribution;
	}

	/**
	 * @return the isOutputTermWeightDistribution
	 */
	public boolean isOutputTermWeightDistribution() {
		return isOutputTermWeightDistribution;
	}

	/**
	 * @param isOutputDocumentWeight
	 *            the isOutputDocumentWeight to set
	 */
	public void setOutputDocumentWeight(boolean isOutputDocumentWeight) {
		this.isOutputDocumentWeight = isOutputDocumentWeight;
	}

	/**
	 * @return the isOutputDocumentWeight
	 */
	public boolean isOutputDocumentWeight() {
		return isOutputDocumentWeight;
	}

	/**
	 * Set sorting method using the following options: Computed Relevance:
	 * DataCollector.SORT_COMPUTED_RELEVANCE Document ID (File name):
	 * DataCollector.SORT_DOCUMENT_ID Document Title:
	 * DataCollector.SORT_DOCUMENT_SIZE Document Size:
	 * DataCollector.SORT_DOCUMENT_TITLE
	 * 
	 * @see ca.uwindsor.cs.liangjie.research.estimation.DataCollector#SORT_COMPUTED_RELEVANCE
	 * @see ca.uwindsor.cs.liangjie.research.estimation.DataCollector#SORT_DOCUMENT_ID
	 * @see ca.uwindsor.cs.liangjie.research.estimation.DataCollector#SORT_DOCUMENT_SIZE
	 * @see ca.uwindsor.cs.liangjie.research.estimation.DataCollector#SORT_DOCUMENT_TITLE
	 * @param sortmethod
	 *            the sorting method to set
	 */
	public void setSortmethod(int sortmethod) {
		this.sortmethod = sortmethod;
	}

	/**
	 * Get the sorting method to use Computed Relevance:
	 * DataCollector.SORT_COMPUTED_RELEVANCE Document ID (File name):
	 * DataCollector.SORT_DOCUMENT_ID Document Title:
	 * DataCollector.SORT_DOCUMENT_SIZE Document Size:
	 * DataCollector.SORT_DOCUMENT_TITLE
	 * 
	 * @see ca.uwindsor.cs.liangjie.research.estimation.DataCollector#SORT_COMPUTED_RELEVANCE
	 * @see ca.uwindsor.cs.liangjie.research.estimation.DataCollector#SORT_DOCUMENT_ID
	 * @see ca.uwindsor.cs.liangjie.research.estimation.DataCollector#SORT_DOCUMENT_SIZE
	 * @see ca.uwindsor.cs.liangjie.research.estimation.DataCollector#SORT_DOCUMENT_TITLE
	 * @return the sortmethod
	 */
	public int getSortmethod() {
		return sortmethod;
	}

	/**
	 * @return the isLimitedReturnTop
	 */
	public boolean isLimitedReturnTop() {
		return isLimitedReturnTop;
	}

	/**
	 * @param isLimitedReturnTop
	 *            the isLimitedReturnTop to set
	 */
	public void setLimitedReturnTop(boolean isLimitedReturnTop) {
		this.isLimitedReturnTop = isLimitedReturnTop;
	}

	/**
	 * @param isReverseSorting
	 *            the isReverseSorting to set
	 */
	public void setReverseSorting(boolean isReverseSorting) {
		this.isReverseSorting = isReverseSorting;
	}

	/**
	 * @return the isReverseSorting
	 */
	public boolean IsReverseSorting() {
		return isReverseSorting;
	}

	/**
	 * Get the exponent of OR.
	 * 
	 * @return exponent of OR
	 */
	public float getExponentofOR() {
		return this.exponentofOR;
	}

	/**
	 * Set the exponent of OR
	 * 
	 * @param e
	 *            the exponent of OR
	 */
	public void setExponnentofOR(float e) {
		this.exponentofOR = e;
	}

	public double getMeanWeight(){
		return meanweight;
	}
}
